'''
	Reads the text for each revision. Removes punctuation, whitespace, stopwords.
	Add the modified text to the database - corresponding to each revision.
'''

import re
import os
import stopwords as sw
#import manipulateText as mT
import config as cg
import database as db


def addText(rev, Text):
	conn = db.connectDb();
	db.addTextTbRev(conn, rev, Text);
	db.closeConn(conn);


def replaceTags(string):
	string = re.sub(r'\<.*?>', ' ', string)
	return string


def readContent(nDir):
	url = nDir + "/text.txt"
	with open(url) as f:
		content = f.readlines()
	
	return content	
	
def printContent(nDir, output):
	url = nDir + "/filteredText.txt"
	f = open(url, 'w')
	f.write(output)
	
	
def listOfDir():
	url = cg.directoryUrl;
	with open(url) as f:
		content = f.readlines()
		
	return content
	
	
def findRevNo(nDir):
	subAdd = nDir.split('/');
	Id = subAdd[5].split('_');
	return Id[1]
	
def filterData():
	dirs = listOfDir()
	i = 1;
	conn = db.connectDb();
	db.modifyTbRev(conn);
	db.closeConn(conn)

	for nDir in dirs:
		
		nDir = nDir.strip('\n')
		print nDir
		
		if ( i == 1 ):
	
			content = readContent(nDir)
			Input = ''
			for item in content:
				Input += item.strip('\n');
			
			output = replaceTags(Input)
			output = output.strip('\t')
			output = sw.removePunc(output)
			output = sw.removeStopWords(output)
			
			rev = findRevNo(nDir);
			addText(rev, output);
		#printContent(nDir, output)
		#return
	
	
filterData()
